# -*- coding: utf-8 -*-
import scrapy
from myspider.items import sightimgitem

class JingQuSpider(scrapy.spiders.Spider):
    name = "jingqutu"
    allowed_domains = ["www.ctrip.com", "you.ctrip.com"]
    start_urls = ["http://you.ctrip.com/sitemap/spotdis/c0"]

    def parse(self, response):
        baseurl = 'http://you.ctrip.com'
        sightlisturl = 'http://you.ctrip.com/sight/'
        sitemapurl = 'http://you.ctrip.com/sitemap/spots/'

        d = {'北京市': '110000','天津市': '120000','河北省': '130000','山西省': '140000','内蒙古自治区': '150000','辽宁省': '210000','吉林省': '220000','黑龙江省': '230000','上海市': '310000','江苏省': '320000','浙江省': '330000','安徽省': '340000','福建省': '350000','江西省': '360000','山东省': '370000','河南省': '410000','湖北省': '420000','湖南省': '430000','广东省': '440000','广西壮族自治区': '450000','海南省': '460000','重庆市': '500000','四川省': '510000','贵州省': '520000','云南省': '530000','西藏自治区': '540000','陕西省': '610000','甘肃省': '620000','青海省': '630000','宁夏回族自治区': '640000','新疆维吾尔自治区': '650000','台湾省': '710000','香港特别行政区': '810000','澳门特别行政区': '820000'}
        d1 = {'北京市': 'beijing1.html',
              '天津市': 'tianjin154.html',
              '上海市': 'shanghai2.html',
              '重庆市': 'chongqing158.html',
              '香港特别行政区': 'hongkong38.html',
              '澳门特别行政区': 'macau39.html'}
        d2 = {'河北省': 'c100059',
              '山西省': 'c100056',
              '内蒙古自治区': 'c100062',
              '辽宁省': 'c100061',
              '吉林省': 'c100031',
              '黑龙江省': 'c100055',
              '江苏省': 'c100066',
              '浙江省': 'c100065',
              '安徽省': 'c100068',
              '福建省': 'c100038',
              '江西省': 'c100054',
              '山东省': 'c100039',
              '河南省': 'c100058',
              '湖北省': 'c100067',
              '湖南省': 'c100053',
              '广东省': 'c100051',
              '广西壮族自治区': 'c100052',
              '海南省': 'c100001',
              '四川省': 'c100009',
              '贵州省': 'c100064',
              '云南省': 'c100007',
              '西藏自治区': 'c100003',
              '陕西省': 'c100057',
              '甘肃省': 'c100060',
              '青海省': 'c100032',
              '宁夏回族自治区': 'c100063',
              '新疆维吾尔自治区': 'c100008'}
        d3 = {'台湾省': 'countrysightlist/taiwan100076.html'}

        #直辖市
        #yield scrapy.Request(sightlisturl + d1['北京市'], callback=lambda response, areacode=d['北京市']:self.parse_page(response,areacode), dont_filter=True)
        #yield scrapy.Request(sightlisturl + d1['天津市'], callback=lambda response, areacode=d['天津市']:self.parse_page(response,areacode), dont_filter=True)
        #yield scrapy.Request(sightlisturl + d1['上海市'], callback=lambda response, areacode=d['上海市']:self.parse_page(response,areacode), dont_filter=True)
        #yield scrapy.Request(sightlisturl + d1['重庆市'], callback=lambda response, areacode=d['重庆市']:self.parse_page(response,areacode), dont_filter=True)
        #yield scrapy.Request(sightlisturl + d1['香港特别行政区'], callback=lambda response, areacode=d['香港特别行政区']:self.parse_page(response,areacode), dont_filter=True)
        #yield scrapy.Request(sightlisturl + d1['澳门特别行政区'], callback=lambda response, areacode=d['澳门特别行政区']:self.parse_page(response,areacode), dont_filter=True)


        #yield scrapy.Request(sitemapurl + d2['河北省'], callback=lambda response, areacode=d['河北省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['山西省'], callback=lambda response, areacode=d['山西省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['内蒙古自治区'], callback=lambda response, areacode=d['内蒙古自治区']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['辽宁省'], callback=lambda response, areacode=d['辽宁省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['吉林省'], callback=lambda response, areacode=d['吉林省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['黑龙江省'], callback=lambda response, areacode=d['黑龙江省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['江苏省'], callback=lambda response, areacode=d['江苏省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['浙江省'], callback=lambda response, areacode=d['浙江省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['安徽省'], callback=lambda response, areacode=d['安徽省']:self.parse_citylist(response,areacode), dont_filter=True)
        #?yield scrapy.Request(sitemapurl + d2['福建省'], callback=lambda response, areacode=d['福建省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['江西省'], callback=lambda response, areacode=d['江西省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['山东省'], callback=lambda response, areacode=d['山东省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['河南省'], callback=lambda response, areacode=d['河南省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['湖北省'], callback=lambda response, areacode=d['湖北省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['湖南省'], callback=lambda response, areacode=d['湖南省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['广东省'], callback=lambda response, areacode=d['广东省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['广西壮族自治区'], callback=lambda response, areacode=d['广西壮族自治区']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['海南省'], callback=lambda response, areacode=d['海南省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['四川省'], callback=lambda response, areacode=d['四川省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['贵州省'], callback=lambda response, areacode=d['贵州省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['云南省'], callback=lambda response, areacode=d['云南省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['西藏自治区'], callback=lambda response, areacode=d['西藏自治区']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['陕西省'], callback=lambda response, areacode=d['陕西省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['甘肃省'], callback=lambda response, areacode=d['甘肃省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['青海省'], callback=lambda response, areacode=d['青海省']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['宁夏回族自治区'], callback=lambda response, areacode=d['宁夏回族自治区']:self.parse_citylist(response,areacode), dont_filter=True)
        #yield scrapy.Request(sitemapurl + d2['新疆维吾尔自治区'], callback=lambda response, areacode=d['新疆维吾尔自治区']:self.parse_citylist(response,areacode), dont_filter=True)

        #还差台湾省
        yield scrapy.Request('http://you.ctrip.com/sightlist/taipeicity360.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/gaoxiong756.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/hualien1366.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/pingtung1364.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/tainan757.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/nantou759.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/taizhong1369.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/newtaipeicity2063.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/taitung760.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/chiayi758.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/yilan1383.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/changhua3180.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/keelung1382.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/jinmen1368.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/taoyuan1439.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/hsinchu1197.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/penghu1367.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/chiayicounty3182.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/miaoli3199.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/matsu1445356.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/yunlin3197.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)
        yield scrapy.Request('http://you.ctrip.com/sightlist/hsinchucounty3198.html', callback=lambda response, areacode='710000':self.parse_page(response,areacode), dont_filter=True)



        #sitemaps = response.xpath('//div[@class="map_title cf"]/a/@href').extract()
        #for sitemap in sitemaps:
        #    yield scrapy.Request( baseurl + sitemap, callback=self.parse_citylist)
        #    pass

        pass

    def parse_citylist(self, response, areacode):
        baseurl = "http://you.ctrip.com"
        links = response.xpath('//ul[@class="map_linklist cf"]/li/a/@href').extract()
        for link in links:
            link = link.replace('sightlist/', 'sight/')
            url = baseurl + link
            yield scrapy.Request(url, callback=lambda response, areacode=areacode:self.parse_page(response,areacode), dont_filter=True)
        pass

    def parse_page(self, response, areacode):
        baseurl = "http://you.ctrip.com"
        sightselectors = response.xpath('//div[@class="list_wide_mod2"]//dt/a')
        for selector in sightselectors:
            url = baseurl + selector.xpath('@href').extract_first()
            yield scrapy.Request(url, callback=lambda response, areacode=areacode:self.parse_sight(response,areacode), dont_filter=True)
            pass
        nextpage = response.xpath('//a[@class="nextpage"]/@href').extract_first()
        if not nextpage is None:
            if nextpage.strip() != '':
                yield scrapy.Request(baseurl + nextpage, callback=lambda response, areacode=areacode:self.parse_page(response,areacode), dont_filter=True)
        pass

    def parse_sight(self, response, areacode):
        # 提取内容
        name = response.xpath('//div[@class="dest_toptitle detail_tt"]//a/text()').extract_first()
        if not name is None:
            name = name.strip()

        desc = response.xpath('//div[@itemprop="description"]').extract_first()
        if desc is None:
            return
        desc = desc.strip()
        if desc == '':
            return

        province = response.xpath('//div[@class="breadbar_v1 cf"]/ul/li[4]/a/text()').extract_first()
        if not province is None:
            province = province.strip()
        if province == '':
            return

        imgurls = response.xpath('//div[@class="carousel-inner"]//img/@src').extract()
        if imgurls is None:
            return
        if len(imgurls)==0:
            return

        item = sightimgitem()
        item['areacode'] = areacode
        item['province'] = province
        item['sight'] = name
        item['imgurls'] = imgurls
        yield item

    pass